package cn.lecosa.spark
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD.rddToPairRDDFunctions
import org.apache.spark.sql.SQLContext


object Wc2 {
  def main(args: Array[String]) {
    val conf = new SparkConf(); //创建SparkConf对象  设置spark参数
    conf.setAppName("wow,My First Spark App!"); //设置应用程序的名称，在spark程序运行的监控界面可以看到名称
    conf.setMaster("local[2]"); //此时，程序在本地运行，不需要安装spark集群
    val sc = new SparkContext(conf);

    val lines = sc.textFile("hdfs://cdh2:8020/testData/word/words.txt", 1); //path:文件路径    minPartitions：最小并行度的数量
    val words =lines.flatMap(x=>x.split(" ")).map(x=>(x,1));
    val sqlContext = new SQLContext(sc);
    import sqlContext.implicits._
    val result =words.reduceByKey(_+_).sortByKey().toDF("word","count");

    				result.show()
				result.printSchema
//    result.foreach(println)
    sc.stop();
  }
}